from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import HtmlXPathSelector
from scrapy.item import Item
from btspider.items import BtInfo
import sys

class MySpider(CrawlSpider):
    name = '54new.com'
    allowed_domains = ['54new.com']
    start_urls = ['http://www.54new.com/seedlist-all-0-1.html']

    rules = (
        # Extract links matching 'category.php' (but not matching 'subsection.php')
        # and follow links from them (since no callback means follow=True by default).
        Rule(SgmlLinkExtractor(allow=('seedlist-all-0-.*', ), )),

        # Extract links matching 'item.php' and parse them with the spider's method parse_item
        Rule(SgmlLinkExtractor(allow=('seed-.*\.html', )), callback='parse_item'),
    )

    def parse_item(self, response):
        self.log('Seed URL: %s' % response.url)

        hxs = HtmlXPathSelector(response)
        item = BtInfo()
        item['title'] = hxs.select('//div[@id="base"]/ul/li/h1/text()').extract()[0]
        item['detail_url'] =  response.url
	item['seed_url'] = hxs.select('//div[@id="base"]/ul/li[6]/a[2]/@href').extract()[0]
	item['file_size'] = hxs.select('//div[@id="base"]/ul/li[4]/text()').extract()[0]
	try:
	    item['bt_type'] = hxs.select('//div[@id="base"]/ul/li[2]/a/text()').extract()[0]
	except:
	    item['bt_type'] = hxs.select('//div[@id="base"]/ul/li[2]/a/span/text()').extract()[0]

        item['publish_time'] = hxs.select('//div[@id="base"]/ul/li[5]/text()').extract()[0]
	print item['title']
	print item['detail_url']
	print item['seed_url']
	print item['file_size']
	print item['bt_type']
	print item['publish_time']
        #item['name'] = hxs.select('//td[@id="item_name"]/text()').extract()
        #item['description'] = hxs.select('//td[@id="item_description"]/text()').extract()
        #return item
